(** 
    Page.ml
    This module deals with parsing page and returns a set of words.
    This module will run in several machines in a parts of the wiki page. 
    The partitioning is managed by an oracle that takes care of splitting the file
    and returns a queue with pages' start position. See page.mli for details.
    Bozman Çagdas <bozman.cagdas@gmail.com> - 2011
    Razakamiadana Tefin <tefy.razaka@gmail.com>   - 2011
*)

let take q = try Some (Queue.take q) with Queue.Empty -> None

(* Parses the current page until the new one and updates the set of word *)
(* let parse_page in_ch set =  (\* XXCAGOXX TODO: ne parse pas bien la ligne *\) *)
(*   let rec aux set = *)
(*     match Mlib.read in_ch with *)
(*       | None -> set *)
(*       | Some p ->  *)
(* 	let new_set = *)
(* 	  if Str.string_match Mlib.link p 0 then  *)
(* 	    Mlib.Set.add (Mlib.from_group p 1) set  *)
(* 	  else set *)
(* 	in  *)
(* 	aux new_set *)
(*   in aux set *)

(** XXCAGOXX TODO: **)
let bfs l = 
  snd (List.fold_left 
	 (fun (b, accu) x -> if b then (false, x :: accu) else (true, accu)) 
	 (true, []) l)
 
let parse_page in_ch set =
  let rec aux set = 
    match Mlib.read_page in_ch with
      | _, None -> set
      | _, Some page ->
	let page = Str.full_split (Str.regexp "\\[\\[\\|\\]\\]") page in
	let l = (Mlib.compact [] page) in (* CHANGE LA LIGNE PAR CELLE LA *)
	let new_set = List.fold_left (fun set x -> Mlib.Set.add x set) set l in
	let _ = List.iter (fun x -> Printf.printf "%s " x) l in
	aux new_set
  in
  aux set

(* Starts reading a new page in setting the current reading position *)
let start_reading_page in_ch page_position set =
  let _ = seek_in in_ch page_position in
  parse_page in_ch set

(* Reads a page and returns a set of new word responding to the request *)
let read_page in_ch q set = 
  (* XXCAGO XX TODO ici on pourrait lancer les page dans des threads et faire l'union 
     des ensembles à la fin *)
  let rec aux q set =
    match take q with
      | None -> set
      | Some page_position -> 
	let new_set = start_reading_page in_ch page_position set in
	aux q new_set
  in
  aux q set
  
let start in_ch q = 
  read_page in_ch q (Mlib.Set.empty)
